{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'paddlehub'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-e143caa33f7d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mcollections\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCounter\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[1;32mimport\u001b[0m \u001b[0mpaddlehub\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mhub\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpaddle\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel_selection\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'paddlehub'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "import paddlehub as hub\n",
    "import paddle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from paddlehub.datasets.base_nlp_dataset import TextClassificationDataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>瞧着这小样儿，突然间感动了，爸妈怎么把我拉扯大的呀～～～</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>习惯和凑和的力量何其强大，革命总是被逼到无法接受的程度以后才会发生，这时仍要忍受数小时的漫长...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5.尽量在7点前起床，这样有利于排出宿便；</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>原来不知道在何时开始，我已经不再是儿童了，不再是那个可以撒娇的小孩子了！</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>宝贝，节日快乐。</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>我们还要这样的阳光吗？</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>据说济南最低温度24度，可今天青岛最高才21度……</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>据说济南最低温度24度，可今天青岛最高才21度……</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>这才是你的舞台啊！！！</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>享受每一刻的感觉，欣赏每一处的风景，这就是人生。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   num                                               text  label\n",
       "0    1                       瞧着这小样儿，突然间感动了，爸妈怎么把我拉扯大的呀～～～    3.0\n",
       "1    2  习惯和凑和的力量何其强大，革命总是被逼到无法接受的程度以后才会发生，这时仍要忍受数小时的漫长...    0.0\n",
       "2    3                              5.尽量在7点前起床，这样有利于排出宿便；    0.0\n",
       "3    4               原来不知道在何时开始，我已经不再是儿童了，不再是那个可以撒娇的小孩子了！    2.0\n",
       "4    5                                           宝贝，节日快乐。    1.0\n",
       "5    6                                        我们还要这样的阳光吗？    5.0\n",
       "6    7                          据说济南最低温度24度，可今天青岛最高才21度……    6.0\n",
       "7    8                          据说济南最低温度24度，可今天青岛最高才21度……    6.0\n",
       "8    9                                        这才是你的舞台啊！！！    5.0\n",
       "9   10                           享受每一刻的感觉，欣赏每一处的风景，这就是人生。    0.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_excel('moods_classify8_unprocessed.xlsx')\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Cannot interpret '<attribute 'dtype' of 'numpy.generic' objects>' as a data type",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-a74c58233b9e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36minfo\u001b[1;34m(self, verbose, buf, max_cols, memory_usage, null_counts)\u001b[0m\n\u001b[0;32m   2272\u001b[0m                         self.index._is_memory_usage_qualified()):\n\u001b[0;32m   2273\u001b[0m                     \u001b[0msize_qualifier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'+'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2274\u001b[1;33m             \u001b[0mmem_usage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmemory_usage\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdeep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdeep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2275\u001b[0m             lines.append(\"memory usage: {mem}\\n\".format(\n\u001b[0;32m   2276\u001b[0m                 mem=_sizeof_fmt(mem_usage, size_qualifier)))\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mmemory_usage\u001b[1;34m(self, index, deep)\u001b[0m\n\u001b[0;32m   2366\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2367\u001b[0m             result = Series(self.index.memory_usage(deep=deep),\n\u001b[1;32m-> 2368\u001b[1;33m                             index=['Index']).append(result)\n\u001b[0m\u001b[0;32m   2369\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2370\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, data, index, dtype, name, copy, fastpath)\u001b[0m\n\u001b[0;32m    273\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m                 data = _sanitize_array(data, index, dtype, copy,\n\u001b[1;32m--> 275\u001b[1;33m                                        raise_cast_failure=True)\n\u001b[0m\u001b[0;32m    276\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSingleBlockManager\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfastpath\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m_sanitize_array\u001b[1;34m(data, index, dtype, copy, raise_cast_failure)\u001b[0m\n\u001b[0;32m   4147\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4148\u001b[0m             subarr = construct_1d_arraylike_from_scalar(\n\u001b[1;32m-> 4149\u001b[1;33m                 value, len(index), dtype)\n\u001b[0m\u001b[0;32m   4150\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4151\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\dtypes\\cast.py\u001b[0m in \u001b[0;36mconstruct_1d_arraylike_from_scalar\u001b[1;34m(value, length, dtype)\u001b[0m\n\u001b[0;32m   1199\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_integer_dtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1200\u001b[0m             \u001b[0mdtype\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1201\u001b[1;33m         \u001b[0msubarr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1202\u001b[0m         \u001b[0msubarr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfill\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1203\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: Cannot interpret '<attribute 'dtype' of 'numpy.generic' objects>' as a data type"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "num      False\n",
       "text      True\n",
       "label     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>48</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>93</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>367</th>\n",
       "      <td>368</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1101</th>\n",
       "      <td>1102</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1112</th>\n",
       "      <td>1113</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1161</th>\n",
       "      <td>1162</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1208</th>\n",
       "      <td>1209</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1249</th>\n",
       "      <td>1250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1694</th>\n",
       "      <td>1695</td>\n",
       "      <td>我想说中国皇帝有上千个老婆呢。</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1705</th>\n",
       "      <td>1706</td>\n",
       "      <td>2、总是和别人比较。</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2141</th>\n",
       "      <td>2142</td>\n",
       "      <td>在最艰难的时刻，更要相信自己手中握有最好的猎枪。</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2188</th>\n",
       "      <td>2189</td>\n",
       "      <td>今日话题，走着：全国高考今日拉开帷幕，各地高考作文题陆续公布。</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2278</th>\n",
       "      <td>2279</td>\n",
       "      <td>今天六一节，感觉没有什么不同的，很多朋友互相问候节日快乐！</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2360</th>\n",
       "      <td>2361</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2361</th>\n",
       "      <td>2362</td>\n",
       "      <td>禁买令最大的问题在于地域性歧视方面，一是户籍问题，而是限家奴而不限友邦。</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2362</th>\n",
       "      <td>2363</td>\n",
       "      <td>蔡健雅唱过，得不到的就更加爱，太容易来的就不理睬。</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4699</th>\n",
       "      <td>4700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4700</th>\n",
       "      <td>4701</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4841</th>\n",
       "      <td>4842</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4855</th>\n",
       "      <td>4856</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4910</th>\n",
       "      <td>4911</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4943</th>\n",
       "      <td>4944</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5005</th>\n",
       "      <td>5006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5069</th>\n",
       "      <td>5070</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10870</th>\n",
       "      <td>10871</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10923</th>\n",
       "      <td>10924</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11108</th>\n",
       "      <td>11109</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11672</th>\n",
       "      <td>11673</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12324</th>\n",
       "      <td>12325</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13252</th>\n",
       "      <td>13253</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14592</th>\n",
       "      <td>14593</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20281</th>\n",
       "      <td>20282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20380</th>\n",
       "      <td>20381</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20478</th>\n",
       "      <td>20479</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20569</th>\n",
       "      <td>20570</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23780</th>\n",
       "      <td>23781</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         num                                  text  label\n",
       "17        18                                   NaN    0.0\n",
       "47        48                                   NaN    0.0\n",
       "92        93                                   NaN    0.0\n",
       "367      368                                   NaN    0.0\n",
       "1101    1102                                   NaN    0.0\n",
       "1112    1113                                   NaN    0.0\n",
       "1161    1162                                   NaN    0.0\n",
       "1208    1209                                   NaN    0.0\n",
       "1249    1250                                   NaN    0.0\n",
       "1694    1695                       我想说中国皇帝有上千个老婆呢。    NaN\n",
       "1705    1706                            2、总是和别人比较。    NaN\n",
       "2141    2142              在最艰难的时刻，更要相信自己手中握有最好的猎枪。    NaN\n",
       "2188    2189       今日话题，走着：全国高考今日拉开帷幕，各地高考作文题陆续公布。    NaN\n",
       "2278    2279         今天六一节，感觉没有什么不同的，很多朋友互相问候节日快乐！    NaN\n",
       "2360    2361                                   NaN    0.0\n",
       "2361    2362  禁买令最大的问题在于地域性歧视方面，一是户籍问题，而是限家奴而不限友邦。    NaN\n",
       "2362    2363             蔡健雅唱过，得不到的就更加爱，太容易来的就不理睬。    NaN\n",
       "4699    4700                                   NaN    0.0\n",
       "4700    4701                                   NaN    0.0\n",
       "4841    4842                                   NaN    0.0\n",
       "4855    4856                                   NaN    0.0\n",
       "4910    4911                                   NaN    0.0\n",
       "4943    4944                                   NaN    0.0\n",
       "5005    5006                                   NaN    0.0\n",
       "5069    5070                                   NaN    0.0\n",
       "10870  10871                                   NaN    0.0\n",
       "10923  10924                                   NaN    1.0\n",
       "11108  11109                                   NaN    5.0\n",
       "11672  11673                                   NaN    0.0\n",
       "12324  12325                                   NaN    0.0\n",
       "13252  13253                                   NaN    5.0\n",
       "14592  14593                                   NaN    5.0\n",
       "20281  20282                                   NaN    0.0\n",
       "20380  20381                                   NaN    0.0\n",
       "20478  20479                                   NaN    0.0\n",
       "20569  20570                                   NaN    0.0\n",
       "23780  23781                                   NaN    5.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.isnull().values==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "num      False\n",
       "text     False\n",
       "label    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dropna(subset=['text','label'],axis=0,how='any',inplace=True)\n",
    "df.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>据说济南最低温度24度，可今天青岛最高才21度……</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>110</td>\n",
       "      <td>地动山摇！</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>155</td>\n",
       "      <td>每一个人应该经常问自己，我们为国为家做过些值得为之骄傲的贡献吗？</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>201</td>\n",
       "      <td>一半在尘土里安详，一半在风里飞扬，一半洒落阴凉，一半沐浴阳光。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>221</td>\n",
       "      <td>……突然觉得……</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>244</td>\n",
       "      <td>1 A　光秃秃的什么都没有 2 B　有部落及热闹的人烟 3 C　有搁浅的船只和一些财物 4 ...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>246</th>\n",
       "      <td>247</td>\n",
       "      <td>快來圍觀\"珠圓玉潤\"的朴先生與女主角又笑又鬧的吃飯演技!!</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248</th>\n",
       "      <td>249</td>\n",
       "      <td>生日happy啊@李茂CaeSaR @吴恙 [花心][蛋糕][蛋糕][蛋糕][心][心][心]</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>276</th>\n",
       "      <td>277</td>\n",
       "      <td>這幾天發得太多風景的照片　但我就是喜愛拍攝這類題材　好吧　就最後一張　雖然今夜的他　沒有前天...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>280</th>\n",
       "      <td>281</td>\n",
       "      <td>总参谋部、总政治部近日联合发出指示，要求全军和武警部队加强教育管理，严格落实有关规定，防范军...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>324</td>\n",
       "      <td>邮递员发布会的时候，听金在中嘴里说出那些话，一瞬间是有些错愕的。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>354</td>\n",
       "      <td>韩国公正交易委员会今日公布了违反音乐供给服务业资格而被处以罚款的15个相关企业的名单，共计罚...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>404</td>\n",
       "      <td>如果有谬误……</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>437</td>\n",
       "      <td>近视眼见了，不知是什么东西，便俯身去拾。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>447</th>\n",
       "      <td>448</td>\n",
       "      <td>明天还要赶到釜山去，可怜的穷孩子只能搭五个小时巴士···伤不起啊！！！</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>449</td>\n",
       "      <td>比如不让农村学生上大学。。</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>460</th>\n",
       "      <td>461</td>\n",
       "      <td>1997年巴西对阵法国，卡洛斯的重炮任意球，轰出了这一道违反了物理4理论的弧线任意球。。。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>462</td>\n",
       "      <td>我说，考古，就是八卦历史。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>523</td>\n",
       "      <td>玩游戏的都知道NPC（Non-Player-Controlled Character ）是系...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593</th>\n",
       "      <td>594</td>\n",
       "      <td>时间在变，有些东西也在变吧。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>625</td>\n",
       "      <td>连欺负丸子哥哥都是一个模式哦~~不可以这样哟~~丸子哥哥可是U的长辈！</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>635</td>\n",
       "      <td>我心中最喜欢的足球俱乐部队。</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>677</th>\n",
       "      <td>678</td>\n",
       "      <td>准备开始打包了，突然发现要带回去的护肤品好多。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>725</th>\n",
       "      <td>726</td>\n",
       "      <td>她是女人，敏感；她是女人，爱吃醋；</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>735</th>\n",
       "      <td>736</td>\n",
       "      <td>毕竟相处时间长了、它俩有感情的！</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>783</th>\n",
       "      <td>784</td>\n",
       "      <td>//@全球奇闻趣事:现在拿出你的手机，看看电话簿里的第7个人，你敢不敢对TA说我爱你？</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>801</th>\n",
       "      <td>802</td>\n",
       "      <td>今天估计没有渔船进港，海鲜小摊寥寥无几。??????????????????????????...</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>892</td>\n",
       "      <td>一个人只有一个心脏，却有两个心房。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>899</td>\n",
       "      <td>考生：就说是临时工，考官：今晚上班。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>910</th>\n",
       "      <td>911</td>\n",
       "      <td>刚琢磨明白，都是戒烟闹的，一天一包的量，现在一根不抽，这罪遭的，憋的慌，烦的慌，抓心挠肺的。</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26432</th>\n",
       "      <td>26433</td>\n",
       "      <td>叶碧花繁春满城，诗酒不让李杜前。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26433</th>\n",
       "      <td>26434</td>\n",
       "      <td>啥啥之后。。。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26434</th>\n",
       "      <td>26435</td>\n",
       "      <td>李书福:企业要听党的话，党啊，我亲爱的奶妈。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26435</th>\n",
       "      <td>26436</td>\n",
       "      <td>NO.9处女。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26436</th>\n",
       "      <td>26437</td>\n",
       "      <td>现任美国驻华大使洪博培将于四月三十号离任，预料他将投入二零一二年总统大选。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26437</th>\n",
       "      <td>26438</td>\n",
       "      <td>上联：百年老店遭遇百般尴尬。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26438</th>\n",
       "      <td>26439</td>\n",
       "      <td>安徽省淮南市田家庵区泉山路泉山村 邮政编码查询 - 邮编库 泉山路泉山村位于安徽省淮南市田家庵区，</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26439</th>\n",
       "      <td>26440</td>\n",
       "      <td>俺滴个心肝脾肺肾乃。。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26440</th>\n",
       "      <td>26441</td>\n",
       "      <td>一会要去干妈家了解填志愿的事。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26441</th>\n",
       "      <td>26442</td>\n",
       "      <td>粥的花头有够多的！</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26442</th>\n",
       "      <td>26443</td>\n",
       "      <td>班集体14号清远漂流，考虑再三，还是推了，感觉很冷淡，还是跟深圳的球友吹水吧，学生的日子已过...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26443</th>\n",
       "      <td>26444</td>\n",
       "      <td>真是走哪里都不安全</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26444</th>\n",
       "      <td>26445</td>\n",
       "      <td>昨日，记者在微博上也看到，此类短信不少。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26445</th>\n",
       "      <td>26446</td>\n",
       "      <td>咱的战争结束了。</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26446</th>\n",
       "      <td>26447</td>\n",
       "      <td>韩寒还不是高考文科状元，新概念作文只是一项比赛而已，《独唱团》和《三重门》都算不了什么。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26447</th>\n",
       "      <td>26448</td>\n",
       "      <td>不还给我！</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26448</th>\n",
       "      <td>26449</td>\n",
       "      <td>我们虽然休息少、但是我们值班多啊！</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26449</th>\n",
       "      <td>26450</td>\n",
       "      <td>人之心胸，多欲则窄，寡欲则宽；人之心境，多欲则忙，寡欲则闲；人之心术，多欲则险，寡欲则平；人...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26450</th>\n",
       "      <td>26451</td>\n",
       "      <td>，有一种态度,叫做&amp;lt;不再犹豫 &amp;gt;。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26451</th>\n",
       "      <td>26452</td>\n",
       "      <td>来吧，都来吧。</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26452</th>\n",
       "      <td>26453</td>\n",
       "      <td>刘嘉玲&amp;amp;梁朝伟。</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26453</th>\n",
       "      <td>26454</td>\n",
       "      <td>到处都是转基因，防不胜防啊，昨天也吃了些豆制品。</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26454</th>\n",
       "      <td>26455</td>\n",
       "      <td>老娘宣布，老道让葵酱喊我回家吃饭！</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26455</th>\n",
       "      <td>26456</td>\n",
       "      <td>身上没有一处不圆，皮肤让肉撑得又白又细。</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26456</th>\n",
       "      <td>26457</td>\n",
       "      <td>你们久等啦！</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26457</th>\n",
       "      <td>26458</td>\n",
       "      <td>目光呆泄、反应迟钝、四肢无力、就差发烧了，自己还美滋滋的说帅的不行！</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26458</th>\n",
       "      <td>26459</td>\n",
       "      <td>每天上班都是相同的路，真的想走不一样的路，有一天下很大的雨，要绕路走，结果那天迷了路......</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26459</th>\n",
       "      <td>26460</td>\n",
       "      <td>转眼间，网络科幻小说《悟空传》已十年了，即使十年后昨晚重读仍让人记忆犹新，最记得玄奘说的一句...</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26460</th>\n",
       "      <td>26461</td>\n",
       "      <td>默默等到点开门后，大叔讥笑我，你怎么又睡过头啊。。。</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26461</th>\n",
       "      <td>26462</td>\n",
       "      <td>我国文化传媒的行业细分又进入了一个新的阶段。</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13537 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         num                                               text  label\n",
       "7          8                          据说济南最低温度24度，可今天青岛最高才21度……    6.0\n",
       "109      110                                              地动山摇！    0.0\n",
       "154      155                   每一个人应该经常问自己，我们为国为家做过些值得为之骄傲的贡献吗？    0.0\n",
       "200      201                    一半在尘土里安详，一半在风里飞扬，一半洒落阴凉，一半沐浴阳光。    0.0\n",
       "220      221                                           ……突然觉得……    0.0\n",
       "243      244  1 A　光秃秃的什么都没有 2 B　有部落及热闹的人烟 3 C　有搁浅的船只和一些财物 4 ...    0.0\n",
       "246      247                      快來圍觀\"珠圓玉潤\"的朴先生與女主角又笑又鬧的吃飯演技!!    1.0\n",
       "248      249    生日happy啊@李茂CaeSaR @吴恙 [花心][蛋糕][蛋糕][蛋糕][心][心][心]    1.0\n",
       "276      277  這幾天發得太多風景的照片　但我就是喜愛拍攝這類題材　好吧　就最後一張　雖然今夜的他　沒有前天...    1.0\n",
       "280      281  总参谋部、总政治部近日联合发出指示，要求全军和武警部队加强教育管理，严格落实有关规定，防范军...    0.0\n",
       "323      324                   邮递员发布会的时候，听金在中嘴里说出那些话，一瞬间是有些错愕的。    0.0\n",
       "353      354  韩国公正交易委员会今日公布了违反音乐供给服务业资格而被处以罚款的15个相关企业的名单，共计罚...    0.0\n",
       "403      404                                            如果有谬误……    0.0\n",
       "436      437                               近视眼见了，不知是什么东西，便俯身去拾。    0.0\n",
       "447      448                明天还要赶到釜山去，可怜的穷孩子只能搭五个小时巴士···伤不起啊！！！    2.0\n",
       "448      449                                      比如不让农村学生上大学。。    4.0\n",
       "460      461      1997年巴西对阵法国，卡洛斯的重炮任意球，轰出了这一道违反了物理4理论的弧线任意球。。。    0.0\n",
       "461      462                                      我说，考古，就是八卦历史。    0.0\n",
       "522      523  玩游戏的都知道NPC（Non-Player-Controlled Character ）是系...    0.0\n",
       "593      594                                     时间在变，有些东西也在变吧。    0.0\n",
       "624      625                连欺负丸子哥哥都是一个模式哦~~不可以这样哟~~丸子哥哥可是U的长辈！    1.0\n",
       "634      635                                     我心中最喜欢的足球俱乐部队。    1.0\n",
       "677      678                            准备开始打包了，突然发现要带回去的护肤品好多。    0.0\n",
       "725      726                                  她是女人，敏感；她是女人，爱吃醋；    0.0\n",
       "735      736                                   毕竟相处时间长了、它俩有感情的！    0.0\n",
       "783      784        //@全球奇闻趣事:现在拿出你的手机，看看电话簿里的第7个人，你敢不敢对TA说我爱你？    0.0\n",
       "801      802  今天估计没有渔船进港，海鲜小摊寥寥无几。??????????????????????????...    2.0\n",
       "891      892                                  一个人只有一个心脏，却有两个心房。    0.0\n",
       "898      899                                 考生：就说是临时工，考官：今晚上班。    0.0\n",
       "910      911     刚琢磨明白，都是戒烟闹的，一天一包的量，现在一根不抽，这罪遭的，憋的慌，烦的慌，抓心挠肺的。    5.0\n",
       "...      ...                                                ...    ...\n",
       "26432  26433                                   叶碧花繁春满城，诗酒不让李杜前。    0.0\n",
       "26433  26434                                            啥啥之后。。。    0.0\n",
       "26434  26435                             李书福:企业要听党的话，党啊，我亲爱的奶妈。    0.0\n",
       "26435  26436                                            NO.9处女。    0.0\n",
       "26436  26437              现任美国驻华大使洪博培将于四月三十号离任，预料他将投入二零一二年总统大选。    0.0\n",
       "26437  26438                                     上联：百年老店遭遇百般尴尬。    0.0\n",
       "26438  26439  安徽省淮南市田家庵区泉山路泉山村 邮政编码查询 - 邮编库 泉山路泉山村位于安徽省淮南市田家庵区，    0.0\n",
       "26439  26440                                        俺滴个心肝脾肺肾乃。。    0.0\n",
       "26440  26441                                    一会要去干妈家了解填志愿的事。    0.0\n",
       "26441  26442                                          粥的花头有够多的！    6.0\n",
       "26442  26443  班集体14号清远漂流，考虑再三，还是推了，感觉很冷淡，还是跟深圳的球友吹水吧，学生的日子已过...    0.0\n",
       "26443  26444                                          真是走哪里都不安全    5.0\n",
       "26444  26445                               昨日，记者在微博上也看到，此类短信不少。    0.0\n",
       "26445  26446                                           咱的战争结束了。    3.0\n",
       "26446  26447       韩寒还不是高考文科状元，新概念作文只是一项比赛而已，《独唱团》和《三重门》都算不了什么。    0.0\n",
       "26447  26448                                              不还给我！    4.0\n",
       "26448  26449                                  我们虽然休息少、但是我们值班多啊！    5.0\n",
       "26449  26450  人之心胸，多欲则窄，寡欲则宽；人之心境，多欲则忙，寡欲则闲；人之心术，多欲则险，寡欲则平；人...    0.0\n",
       "26450  26451                            ，有一种态度,叫做&lt;不再犹豫 &gt;。    0.0\n",
       "26451  26452                                            来吧，都来吧。    4.0\n",
       "26452  26453                                       刘嘉玲&amp;梁朝伟。    9.0\n",
       "26453  26454                           到处都是转基因，防不胜防啊，昨天也吃了些豆制品。    5.0\n",
       "26454  26455                                  老娘宣布，老道让葵酱喊我回家吃饭！    3.0\n",
       "26455  26456                               身上没有一处不圆，皮肤让肉撑得又白又细。    1.0\n",
       "26456  26457                                             你们久等啦！    3.0\n",
       "26457  26458                 目光呆泄、反应迟钝、四肢无力、就差发烧了，自己还美滋滋的说帅的不行！    2.0\n",
       "26458  26459   每天上班都是相同的路，真的想走不一样的路，有一天下很大的雨，要绕路走，结果那天迷了路......    6.0\n",
       "26459  26460  转眼间，网络科幻小说《悟空传》已十年了，即使十年后昨晚重读仍让人记忆犹新，最记得玄奘说的一句...    0.0\n",
       "26460  26461                         默默等到点开门后，大叔讥笑我，你怎么又睡过头啊。。。    5.0\n",
       "26461  26462                             我国文化传媒的行业细分又进入了一个新的阶段。    0.0\n",
       "\n",
       "[13537 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.duplicated('text')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop_duplicates(subset='text',keep='first',inplace=True)\n",
    "df.duplicated('text').any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAADuCAYAAAAZZe3jAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAACXBJREFUeJzt3V+o3/ddx/HX2x7LbLQmI2fFpo2pQwbTm2yH+WfgsA3+bTbQCZVtUHXGi25OqchEsVP0rhMFg3js5o3FXnS9WMbQub/gTdlJO9i6KIzONX/qesrSKb2pwY8XSXe6eE7ON8n57uSd83hAaH6/880vrxPoky/f3+93fjXGCAB9fNd2DwDg8gg3QDPCDdCMcAM0I9wAzQg3QDPCDdCMcAM0I9wAzSzM8aB79+4dBw4cmOOhAa5Lx48ff36MsTjl2FnCfeDAgaysrMzx0ADXpar62tRjXSoBaEa4AZoRboBmhBugGeEGaEa4AZoRbnaGQ4eSqrVfhw5t9yK4YsLNzvCpT136NjQi3ADNCDdAM8LNznDXXZe+DY3M8rNK4JrzyU9u9wLYMs64AZoRboBmhBugGeEGaEa4AZoRboBmhBugGeEGaEa4AZoRboBmhBugGeEGaEa4AZoRboBmhBugmUnhrqrfraqnqupLVfWPVfWquYcBsL5Nw11V+5L8dpKlMcaPJrkhyT1zDwNgfVMvlSwk+Z6qWkhyU5Iz800C4FI2DfcY43SSB5M8k+TZJN8cY3zi4uOq6khVrVTVyurq6tYvBSDJtEsle5K8LckdSW5Nsquq3nnxcWOM5THG0hhjaXFxceuXApBk2qWSQ0m+OsZYHWP8T5LHkvzkvLMA2MiUcD+T5Mer6qaqqiR3JTkx7ywANjLlGvfjSR5N8kSSL174M8sz7wJgAwtTDhpjPJDkgZm3ADCBd04CNCPcAM0IN0Azwg3QjHADNCPcAM0IN0Azwg3QjHADNCPcAM0IN0Azwg3QjHADNCPcAM1M+rGu0F7V/79vjO/8DtgCzrgBmhFugGaEG6AZ4QZoxpOT7AyeiOQ64owboBnhBmhGuAGaEW6AZoQboBnhBmhGuAGaEW6AZoQboBnhBmhGuAGaEW6AZoQboBnhBmhmUrirandVPVpV/1ZVJ6rqJ+YeBsD6pv487r9K8k9jjLdX1Y1JbppxEwCXsGm4q+rmJD+V5N4kGWO8lOSleWfBFvMp71xHplwq+aEkq0n+vqqerKqHqmrXzLsA2MCUcC8keUOSvxljHEzyYpL3X3xQVR2pqpWqWlldXd3imQC8bEq4TyU5NcZ4/MLtR3M+5N9mjLE8xlgaYywtLi5u5UYAXmHTcI8x/jPJyap63YW77kry5VlXAbChqa8qeW+Shy+8ouTpJL823ySYgSciuY5MCvcY4wtJlmbeAsAE3jkJ0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0MzUjy6jsara7glco4aPdGtJuHcA/3OuqSr/HrTnUglAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNTA53Vd1QVU9W1cfmHATApV3OGff7kpyYawgA00wKd1XdluQXkzw07xwANjP1jPsvk/x+kv+dcQsAE2wa7qq6O8lzY4zjmxx3pKpWqmpldXV1ywYC8O2mnHG/Oclbq+o/kjyS5M6q+oeLDxpjLI8xlsYYS4uLi1s8E4CXbRruMcYfjDFuG2McSHJPkk+PMd45+zIA1uV13ADNLFzOwWOMzyb57CxLAJjEGTdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzQj3ADNCDdAM8IN0IxwAzSzabir6vaq+kxVnaiqp6rqfd+JYQCsb2HCMeeS3D/GeKKqvi/J8ar6lzHGl2feBsA6Nj3jHmM8O8Z44sLv/zvJiST75h4GwPqmnHF/S1UdSHIwyePrfO1IkiNJsn///i2YdnVuvW1/nj19crtncA2qqu2ewDXmB/bdnjOnntnuGZNNDndVfW+SjyT5nTHGf1389THGcpLlJFlaWhpbtvAKPXv6ZH7mTz+23TOABj7xx3dv94TLMulVJVX13Tkf7YfHGI/NOwmAS5nyqpJK8qEkJ8YYfzH/JAAuZcoZ95uTvCvJnVX1hQu/fmHmXQBsYNNr3GOMf03i2RyAa4R3TgI0I9wAzQg3QDPCDdCMcAM0I9wAzQg3QDPCDdCMcAM0I9zsGOcWVvP893845xZWt3sKXBXhZkc4t7Ca5/YczXNPfT7P7Tkq3rQm3Fz3Xo726Ueezqnlkzn9yNPiTWvCzXXvhV3H8sKTX8/Zz30jSXL2c9/IC09+PS/sOrbNy+DKCDfXvd0vHs7ug7dkz1tenSTZ85ZXZ/fBW7L7xcPbvAyuzGV95iR0tHBuMa85e19yz9Hset2u7D54S15z9r4snFvc7mlwRYSbHeHleN/4I8ey++xh0aY14WbHWDi3mL3f/PXtngFXzTVugGaEG6AZ4QZoRrgBmhFugGaEG6AZ4QZoRrgBmhFugGaEG6AZ4QZoRrgBmhFugGaEG6AZ4QZoRrgBmhFugGaEG6CZSeGuqp+rqn+vqq9U1fvnHgXAxjYNd1XdkORokp9P8vokv1pVr597GADrm3LG/aYkXxljPD3GeCnJI0neNu8sADYy5VPe9yU5+Yrbp5L82MUHVdWRJEeSZP/+/Vsy7mo8eO/Nuf+1d3/r9hv/8Px/j//52jEf+EjyJ48lp/86uXXPha9/NVn6o+RvfyM5cufasbfel7zxjuTY763dd+Sh5O8+k4yH1+479kTy1g8mH70/OfyGtfvrHclv/nSy/O61+w4/eP7vO3N07b7lTye/9aFk5c/O/31JcuZssu89yQO/lHzgl9eO9T35nnxPW/M9ffDem9NJjTEufUDVryT52THGuy/cfleSN40x3rvRn1laWhorKytbOhTgelZVx8cYS1OOnXKp5FSS219x+7YkZ65kGABXb0q4P5/kh6vqjqq6Mck9ST467ywANrLpNe4xxrmqek+Sf05yQ5IPjzGemn0ZAOua8uRkxhgfT/LxmbcAMIF3TgI0I9wAzQg3QDPCDdDMpm/AuaIHrVpN8rUtf2C4enuTPL/dI2AdPzjGWJxy4CzhhmtVVa1MfXcaXKtcKgFoRrgBmhFudprl7R4AV8s1boBmnHEDNCPcAM0IN0Azwg3QjHADNPN/2EhBbrLr5d4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.boxplot(x=df.label,\n",
    "           whis=1.5,\n",
    "           widths=0.8,\n",
    "           patch_artist=True,\n",
    "           showmeans=True,\n",
    "           boxprops={'facecolor':'steelblue'},\n",
    "           flierprops={'markerfacecolor':'red','markeredgecolor':'red','markersize':4},\n",
    "           meanprops={'marker':'D','markerfacecolor':'black','markersize':4},\n",
    "           medianprops={'linestyle':'--','color':'orange'},\n",
    "           labels=[''])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({6.0: 306, 7.0: 112, 9.0: 9})\n"
     ]
    }
   ],
   "source": [
    "Q1=df.label.quantile(q=0.25)\n",
    "Q3=df.label.quantile(q=0.75)\n",
    "low_whisker=Q1-1.5*(Q3-Q1)\n",
    "up_whisker=Q3+1.5*(Q3-Q1)\n",
    "df2=df.label[(df.label>up_whisker)|(df.label<low_whisker)]\n",
    "print(Counter(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1758</th>\n",
       "      <td>1759</td>\n",
       "      <td>让我们的生活一直有新鲜感。</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2503</th>\n",
       "      <td>2504</td>\n",
       "      <td>所以不要那么轻易许下承诺。</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2618</th>\n",
       "      <td>2619</td>\n",
       "      <td>我手头刚好有本1947年11月第一版的《南海诸岛地理志略》，下为插图</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11654</th>\n",
       "      <td>11655</td>\n",
       "      <td>没什么原因，也许只是一个温和的笑容，一句关切的问候。</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11863</th>\n",
       "      <td>11864</td>\n",
       "      <td>（中国新闻网）</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11950</th>\n",
       "      <td>11951</td>\n",
       "      <td>精英大赛新店赛与复活赛即将拉开序幕，来自七个大区的复活选手以及10家店的选手将争夺总决赛最后...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12038</th>\n",
       "      <td>12039</td>\n",
       "      <td>4、缅甸国家队夺魁。</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12131</th>\n",
       "      <td>12132</td>\n",
       "      <td>康康女两岁左右。</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22197</th>\n",
       "      <td>22198</td>\n",
       "      <td>接龙第四棒还请浙江金承龙（全经联新晋副秘书长）发表观点。</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         num                                               text  label\n",
       "1758    1759                                      让我们的生活一直有新鲜感。    9.0\n",
       "2503    2504                                      所以不要那么轻易许下承诺。    9.0\n",
       "2618    2619                 我手头刚好有本1947年11月第一版的《南海诸岛地理志略》，下为插图    9.0\n",
       "11654  11655                         没什么原因，也许只是一个温和的笑容，一句关切的问候。    9.0\n",
       "11863  11864                                            （中国新闻网）    9.0\n",
       "11950  11951  精英大赛新店赛与复活赛即将拉开序幕，来自七个大区的复活选手以及10家店的选手将争夺总决赛最后...    9.0\n",
       "12038  12039                                         4、缅甸国家队夺魁。    9.0\n",
       "12131  12132                                           康康女两岁左右。    9.0\n",
       "22197  22198                       接龙第四棒还请浙江金承龙（全经联新晋副秘书长）发表观点。    9.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['label']==9.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop((df[df['label']==9.0]).index,inplace=True)\n",
    "(df['label']==9.0).any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Cannot interpret '<attribute 'dtype' of 'numpy.generic' objects>' as a data type",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-14-a74c58233b9e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36minfo\u001b[1;34m(self, verbose, buf, max_cols, memory_usage, null_counts)\u001b[0m\n\u001b[0;32m   2272\u001b[0m                         self.index._is_memory_usage_qualified()):\n\u001b[0;32m   2273\u001b[0m                     \u001b[0msize_qualifier\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'+'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2274\u001b[1;33m             \u001b[0mmem_usage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmemory_usage\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdeep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdeep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2275\u001b[0m             lines.append(\"memory usage: {mem}\\n\".format(\n\u001b[0;32m   2276\u001b[0m                 mem=_sizeof_fmt(mem_usage, size_qualifier)))\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mmemory_usage\u001b[1;34m(self, index, deep)\u001b[0m\n\u001b[0;32m   2366\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2367\u001b[0m             result = Series(self.index.memory_usage(deep=deep),\n\u001b[1;32m-> 2368\u001b[1;33m                             index=['Index']).append(result)\n\u001b[0m\u001b[0;32m   2369\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2370\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, data, index, dtype, name, copy, fastpath)\u001b[0m\n\u001b[0;32m    273\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m                 data = _sanitize_array(data, index, dtype, copy,\n\u001b[1;32m--> 275\u001b[1;33m                                        raise_cast_failure=True)\n\u001b[0m\u001b[0;32m    276\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSingleBlockManager\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfastpath\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m_sanitize_array\u001b[1;34m(data, index, dtype, copy, raise_cast_failure)\u001b[0m\n\u001b[0;32m   4147\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4148\u001b[0m             subarr = construct_1d_arraylike_from_scalar(\n\u001b[1;32m-> 4149\u001b[1;33m                 value, len(index), dtype)\n\u001b[0m\u001b[0;32m   4150\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4151\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\dtypes\\cast.py\u001b[0m in \u001b[0;36mconstruct_1d_arraylike_from_scalar\u001b[1;34m(value, length, dtype)\u001b[0m\n\u001b[0;32m   1199\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_integer_dtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1200\u001b[0m             \u001b[0mdtype\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfloat64\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1201\u001b[1;33m         \u001b[0msubarr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlength\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1202\u001b[0m         \u001b[0msubarr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfill\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1203\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: Cannot interpret '<attribute 'dtype' of 'numpy.generic' objects>' as a data type"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    12878.000000\n",
       "mean        28.294378\n",
       "std         22.379213\n",
       "min          1.000000\n",
       "25%         13.000000\n",
       "50%         22.000000\n",
       "75%         37.000000\n",
       "max        320.000000\n",
       "Name: text, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text'].str.len().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'train_test_split' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-0207fbf9b235>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mtrain_labled\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'label'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'text'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtrain_test_split\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_labled\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtext_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mrandom_state\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2021\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'train.txt'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFlase\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mheader\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFlase\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'\\t'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'test.txt'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFlase\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mheader\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFlase\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'\\t'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mtxt_list\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'train.txt'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'test.txt'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'train_test_split' is not defined"
     ]
    }
   ],
   "source": [
    "train_labled=df[['label','text']]\n",
    "train,test=train_test_split(train_labled,text_size=0.2,random_state=2021)\n",
    "train.to_csv('train.txt',index=Flase,header=Flase,sep='\\t')\n",
    "test.to_csv('test.txt',index=Flase,header=Flase,sep='\\t')\n",
    "txt_list=['train.txt','test.txt']\n",
    "l=0\n",
    "for file in txt_list:\n",
    "    with open(file,'r')as f:\n",
    "        l+=len(f.readlines())\n",
    "print('拆分后的数量为：',l)\n",
    "from paddlehub.datasets.base_nlp_dataset import TextClassificationDataset\n",
    "class MyDataset(TextClassificationDataset):\n",
    "    base_path='data'\n",
    "    label_list=['0.0','1.0','2.0','3.0','4.0','5.0','6.0','7.0']\n",
    "    def __init__(self,tokenizer,max_seq_len:int=128,mode:str='train'):\n",
    "        if mode=='train':\n",
    "            data_file='train.txt'\n",
    "        elif mode=='test':\n",
    "            data_file='text.txt'\n",
    "        else:\n",
    "            data_file='dev.txt'\n",
    "        super().__init__(\n",
    "        base_path=self.base_path,\n",
    "        tokenizer=tokenizer,\n",
    "        max_seq_len=max_seq_len,\n",
    "        mode=mode,\n",
    "        data_file=data_file,\n",
    "        label_list=self.label_list,\n",
    "        is_file_with_header=False)\n",
    "model=hub.Module(name='ernie_tiny',task='seq-cls',num_classes=len(MyDataset.label_list))\n",
    "tokenizer=model.get_tokenizer()\n",
    "train_dataset=MyDataset(tokenizer)\n",
    "test_dataset=MyDataset(tokenizer,mode='test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
